import pickle
import helpsk as hlp
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from sklearn.datasets import fetch_openml
import seaborn as sns
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder # , LabelEncoder
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
# from sklearn.base import BaseEstimator, TransformerMixin
%matplotlib inline
# with open('../X_train.pkl', 'rb') as handle:
# X_train = pickle.load(handle)
# print(X_train.shape)
# with open('../y_train.pkl', 'rb') as handle:
# y_train = pickle.load(handle)
# print(len(y_train))
with open('X_test.pkl', 'rb') as handle:
X_test = pickle.load(handle)
print(X_test.shape)
with open('y_test.pkl', 'rb') as handle:
y_test = pickle.load(handle)
print(len(y_test))
(200, 20) 200
X_test.head()
| checking_status | duration | credit_history | purpose | credit_amount | savings_status | employment | installment_commitment | personal_status | other_parties | residence_since | property_magnitude | age | other_payment_plans | housing | existing_credits | job | num_dependents | own_telephone | foreign_worker | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 521 | <0 | 18.0 | existing paid | radio/tv | 3190.0 | <100 | 1<=X<4 | 2.0 | female div/dep/mar | none | 2.0 | real estate | 24.0 | none | own | 1.0 | skilled | 1.0 | none | yes |
| 737 | <0 | 18.0 | existing paid | new car | 4380.0 | 100<=X<500 | 1<=X<4 | 3.0 | male single | none | 4.0 | car | 35.0 | none | own | 1.0 | unskilled resident | 2.0 | yes | yes |
| 740 | <0 | 24.0 | all paid | new car | 2325.0 | 100<=X<500 | 4<=X<7 | 2.0 | male single | none | 3.0 | car | 32.0 | bank | own | 1.0 | skilled | 1.0 | none | yes |
| 660 | >=200 | 12.0 | existing paid | radio/tv | 1297.0 | <100 | 1<=X<4 | 3.0 | male mar/wid | none | 4.0 | real estate | 23.0 | none | rent | 1.0 | skilled | 1.0 | none | yes |
| 411 | no checking | 33.0 | critical/other existing credit | used car | 7253.0 | <100 | 4<=X<7 | 3.0 | male single | none | 2.0 | car | 35.0 | none | own | 2.0 | high qualif/self emp/mgmt | 1.0 | yes | yes |
y_test[0:10]
array([1, 0, 0, 0, 0, 0, 0, 0, 0, 0])
with open('Logistic Regression/test_set_predictions.pkl', 'rb') as handle:
logistic_predictions = pickle.load(handle)
logistic_predictions[0:10]
array([0.40677117, 0.50960321, 0.61519753, 0.32037735, 0.07595768,
0.36274643, 0.0779174 , 0.4667194 , 0.0829285 , 0.11700064])
with open('XGBoost/test_set_predictions.pkl', 'rb') as handle:
xgb_predictions = pickle.load(handle)
xgb_predictions[0:10]
array([0.33118248, 0.59089786, 0.6703654 , 0.39817235, 0.07021371,
0.41777927, 0.06150061, 0.45194232, 0.10988432, 0.21019126],
dtype=float32)
with open('Stacking/test_set_predictions.pkl', 'rb') as handle:
stacking_predictions = pickle.load(handle)
stacking_predictions[0:10]
array([0.49853447, 0.50046754, 0.5013185 , 0.49942148, 0.4958812 ,
0.4995596 , 0.4958812 , 0.49975598, 0.49589697, 0.49672192],
dtype=float32)
from helpsk.sklearn_eval import TwoClassModelComparison
predictions = {
'Logistic Regression': logistic_predictions,
'XGBoost': xgb_predictions,
'Stacking': stacking_predictions,
}
comparison = TwoClassModelComparison(
actual_values=y_test,
predicted_scores=predictions,
)
comparison.all_metrics_df(
dummy_classifier_strategy=['prior', 'constant'],
return_style=True,
round_by=2
)
| AUC | F1 Score | True Positive Rate | True Negative Rate | False Positive Rate | False Negative Rate | Positive Predictive Value | Negative Predictive Value | |
|---|---|---|---|---|---|---|---|---|
| Logistic Regression | 0.80 | 0.59 | 0.51 | 0.91 | 0.09 | 0.49 | 0.70 | 0.82 |
| XGBoost | 0.81 | 0.61 | 0.51 | 0.94 | 0.06 | 0.49 | 0.77 | 0.82 |
| Stacking | 0.80 | 0.57 | 0.47 | 0.91 | 0.09 | 0.53 | 0.70 | 0.81 |
| Dummy (prior) | 0.50 | 0.00 | 0.00 | 1.00 | 0.00 | 1.00 | 0.00 | 0.70 |
| Dummy (constant) | 0.50 | 0.46 | 1.00 | 0.00 | 1.00 | 0.00 | 0.30 | 0.00 |
comparison.plot_metrics_comparison()
comparison.plot_roc_curves()